from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np
import heapq

print "Program Begins..."
h = []

# step-2: load the info for the doc AND docID converter
localDocIDANDUniversalDocIDDict = {}
# the input file format:
# 1st column: trecID
# 2ed column: docID
# 3th column: # of postings have been recorded for this document
inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_WholeIndexUniversalDocID_MappingTableForBabyIndex_sortedByTrecID"
inputFileHandler = open(inputFileName1,"r")
for currentLine in inputFileHandler.readlines():
    currentLineElements = currentLine.strip().split(" ") 
    currentLocalDocID = currentLineElements[1]
    currentUniversalDocID = currentLineElements[2]
    if currentLocalDocID not in localDocIDANDUniversalDocIDDict:
        localDocIDANDUniversalDocIDDict[currentLocalDocID] = currentUniversalDocID
print "len(localDocIDANDUniversalDocIDDict):",len(localDocIDANDUniversalDocIDDict)
print "localDocIDANDUniversalDocIDDict['0']:",localDocIDANDUniversalDocIDDict['0']
inputFileHandler.close()
# exit(1)

# step-1: load the info for term AND termID converter 
termANDTermIDDict = {}
# the input format of the file should be the following:
# termID
# term
# 1.7M terms in the lexicon
# It takes 3.817s to load
inputFileName2 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/TermsWithTermIDFor90778ImportantAndFakeDocsFromGov2"
# It takes 2 mins to load
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithTermID"
inputFileHandler = open(inputFileName2,"r")
currentLine = inputFileHandler.readline()
while currentLine:
    currentLineElements = currentLine.strip().split(" ")
    currentTermID = currentLineElements[0]
    currentTerm = currentLineElements[1]
    if currentTerm not in termANDTermIDDict:
        termANDTermIDDict[currentTerm] = currentTermID
    currentLine = inputFileHandler.readline()
print "len(termANDTermIDDict): ",len(termANDTermIDDict)
print "termANDTermIDDict['0000000000000000']: ",termANDTermIDDict['0000000000000000']
inputFileHandler.close()
# exit(1)


# step0: load the info for all the related TOP10 postings of the final 5K queries
# It takes 1s
top10RelatedPostingsDict = {}
# The input format of the file
# queryID
# docID
# term
# (Just guess) rank in the list for this posting
# pieceNumIndex
inputFileName3 = "/home/diaosi/web-search-engine-wei/polyIRIndexer/tail5KResultsWithRelatedPieceNum_sortedByQID"
inputFileHandler = open(inputFileName3,"r")
currentLine = inputFileHandler.readline()
while currentLine:
    currentLineElements = currentLine.strip().split(" ")
    currentTerm = currentLineElements[2]
    currentDocID = currentLineElements[1]
    currentTermID = 50000000
    if currentTerm in termANDTermIDDict:
        currentTermID = termANDTermIDDict[currentTerm]
    currentPostingKey = currentTermID + "_" + currentDocID 
    if currentPostingKey not in top10RelatedPostingsDict:
        top10RelatedPostingsDict[currentPostingKey] = 1
    currentLine = inputFileHandler.readline()
# print "top10RelatedPostingsDict['bronx_21911485']: ",top10RelatedPostingsDict['bronx_21911485']
print "len(top10RelatedPostingsDict):",len(top10RelatedPostingsDict)
# print top10RelatedPostingsDict
inputFileHandler.close()
# exit(1)



# step1: some very simple popping methods to do it
# # of documents included
# 90778
NUM_OF_DOCUMENTS_INCLUDED = 91000 # the num should be larger than the 90778 document set
inputFileName4 = "/home/diaosi/web-search-engine-wei/polyIRIndexer/selectedDocumentPrefixValuesInfo20131107Night_weight_10_WITH_TIME_COST_Added_2"
inputFileHandler = open(inputFileName4,"r")
lineCounter = 0

currentLine = inputFileHandler.readline()
lineCounter += 1

while currentLine:
    currentDocList1 = []
    currentLineElements = currentLine.strip().split(" ")
    currentLocalDocID = currentLineElements[0]
    currentUniversalDocID = localDocIDANDUniversalDocIDDict[currentLocalDocID]
    numOfTuplesNeededToRead = int( currentLineElements[1] )
    upperBound = numOfTuplesNeededToRead * 2 + 2
    for i in range(2, upperBound, 2):
        currentTuple = (-float( currentLineElements[i+1] ), int( currentLineElements[i] ), currentUniversalDocID)
        # print "currentTuple: ",i,currentTuple
        currentDocList1.append( currentTuple )
        heapq.heappush(h, currentTuple)
    
    currentLine = inputFileHandler.readline()
    lineCounter += 1
    
    if lineCounter % 1000 == 0:
        print "lineCounter processed: ",lineCounter
    
    if lineCounter == NUM_OF_DOCUMENTS_INCLUDED:
        break

print "# of documents included: ",lineCounter - 1
print "# of postings included: ",len(h)
inputFileHandler.close()
# exit(1)

# step2: an evaluation mechanism
NUM_POSTINGS_WANT_TO_POPPED = len(h) # this variable can be changed
NUM_POSTINGS_IN_HEAP = len(h)
NUM_POSTINGS_IN_HEAP_90Percent = int(NUM_POSTINGS_IN_HEAP * 0.9)
NUM_POSTINGS_IN_HEAP_80Percent = int(NUM_POSTINGS_IN_HEAP * 0.8)
NUM_POSTINGS_IN_HEAP_70Percent = int(NUM_POSTINGS_IN_HEAP * 0.7)
NUM_POSTINGS_IN_HEAP_60Percent = int(NUM_POSTINGS_IN_HEAP * 0.6)
NUM_POSTINGS_IN_HEAP_50Percent = int(NUM_POSTINGS_IN_HEAP * 0.5)
NUM_POSTINGS_IN_HEAP_40Percent = int(NUM_POSTINGS_IN_HEAP * 0.4)
NUM_POSTINGS_IN_HEAP_30Percent = int(NUM_POSTINGS_IN_HEAP * 0.3)
NUM_POSTINGS_IN_HEAP_20Percent = int(NUM_POSTINGS_IN_HEAP * 0.2)
NUM_POSTINGS_IN_HEAP_10Percent = int(NUM_POSTINGS_IN_HEAP * 0.1)
NUM_POSTINGS_IN_HEAP_5Percent = int(NUM_POSTINGS_IN_HEAP * 0.05)
NUM_POSTINGS_IN_HEAP_1Percent = int(NUM_POSTINGS_IN_HEAP * 0.01)
print "NUM_POSTINGS_IN_HEAP_1Percent:",NUM_POSTINGS_IN_HEAP_1Percent
print "NUM_POSTINGS_IN_HEAP_5Percent:",NUM_POSTINGS_IN_HEAP_5Percent
print "NUM_POSTINGS_IN_HEAP_10Percent:",NUM_POSTINGS_IN_HEAP_10Percent
print "NUM_POSTINGS_IN_HEAP_20Percent:",NUM_POSTINGS_IN_HEAP_20Percent
print "NUM_POSTINGS_IN_HEAP_30Percent:",NUM_POSTINGS_IN_HEAP_30Percent
print "NUM_POSTINGS_IN_HEAP_40Percent:",NUM_POSTINGS_IN_HEAP_40Percent
print "NUM_POSTINGS_IN_HEAP_50Percent:",NUM_POSTINGS_IN_HEAP_50Percent
print "NUM_POSTINGS_IN_HEAP_60Percent:",NUM_POSTINGS_IN_HEAP_60Percent
print "NUM_POSTINGS_IN_HEAP_70Percent:",NUM_POSTINGS_IN_HEAP_70Percent
print "NUM_POSTINGS_IN_HEAP_80Percent:",NUM_POSTINGS_IN_HEAP_80Percent
print "NUM_POSTINGS_IN_HEAP_90Percent:",NUM_POSTINGS_IN_HEAP_90Percent
NUM_POSTINGS_POPPED_ARE_IN_TOP10 = 0
NUM_POSTINGS_CAN_BE_POPPED = 0
if NUM_POSTINGS_WANT_TO_POPPED > NUM_POSTINGS_IN_HEAP:
    NUM_POSTINGS_CAN_BE_POPPED = NUM_POSTINGS_IN_HEAP
else:
    NUM_POSTINGS_CAN_BE_POPPED = NUM_POSTINGS_WANT_TO_POPPED

print "NUM_POSTINGS_CAN_BE_POPPED: ",NUM_POSTINGS_CAN_BE_POPPED
for i in range(0,NUM_POSTINGS_CAN_BE_POPPED):
    # print i
    currentTupleBeingPopped = heapq.heappop(h)
    (utilityValue,termIDInIntFormat,docIDInStringFormat) = currentTupleBeingPopped
    termIDInStringFormat = str(termIDInIntFormat)
    currentKey = termIDInStringFormat + "_" + docIDInStringFormat
    if currentKey in top10RelatedPostingsDict:
        print "Posting_ID: ",termIDInStringFormat,docIDInStringFormat
        NUM_POSTINGS_POPPED_ARE_IN_TOP10 += 1
    else:
        pass
        # print "the currentKey",currentKey,"is NOT in the top10RelatedPostingsDict."
        # exit(1)
    if i == NUM_POSTINGS_IN_HEAP_1Percent - 1:
        print "1%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_5Percent - 1:
        print "5%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_10Percent - 1:
        print "10%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_20Percent - 1:
        print "20%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_30Percent - 1:
        print "30%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_40Percent - 1:
        print "40%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_50Percent - 1:
        print "50%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_60Percent - 1:
        print "60%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_70Percent - 1:
        print "70%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_80Percent - 1:
        print "80%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP_90Percent - 1:
        print "90%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    if i == NUM_POSTINGS_IN_HEAP - 1:
        print "100%:",NUM_POSTINGS_POPPED_ARE_IN_TOP10
    
print "NUM_POSTINGS_POPPED_ARE_IN_TOP10: ",NUM_POSTINGS_POPPED_ARE_IN_TOP10
print "Program Ends."

